package com.wontube.smr.bo.sites.youtube.spider;

import java.io.IOException;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.UUID;

import org.apache.ibatis.session.SqlSession;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.wontube.client.extension.utils.DateUtils;
import com.wontube.client.extension.utils.NumberUtils;
import com.wontube.client.extension.utils.StringUtils;
import com.wontube.client.extension.utils.net.URLUtils;
import com.wontube.core.dao.mybatis.QueryExample;
import com.wontube.smr.bo.constants.SiteConstants;
import com.wontube.smr.bo.sites.youtube.YoutubeMovieGenres;
import com.wontube.smr.client.model.po.Movie;
import com.wontube.smr.client.model.po.MovieGenre;
import com.wontube.smr.dao.mapper.MovieGenreMapper;
import com.wontube.smr.dao.mapper.MovieMapper;
import com.wontube.smr.pub.DefaultDBHelper;

/**
 * @author tanggf(2012-8-21)
 *
 */
public final class YoutubeMovieSpider {	
	private final static Logger logger = LoggerFactory.getLogger(YoutubeMovieSpider.class);
	private final static int CONNECT_READ_TIMEOUT = 30 * 1000;
	private final static String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; rv:13.0) Gecko/20100101 Firefox/13.0.1";
	private final static String REFERRER = "http://www.youtube.com/";
	
	private final static String[] MOVIE_CATES = {"action-adventure", "comedy", "animation", 
		"drama", "family", "documentary", "horror", "science-fiction", "romance", "crime",
		"mystery-suspense", "classics"};	
	
	private final static String MOVIE_LIST_URL_FORMAT = "http://www.youtube.com/movies/{}/videos?flow=grid&view=11&page={}";
	private final static String MOVIE_DETAIL_URL_FORMAT = "http://www.youtube.com/movie?v={}";
	private final static String MOVIE_WATCH_URL_FORMAT = "http://www.youtube.com/watch?v={}";
	private final static int MOVIE_MAX_PAGE = 15;
	
	private static Map<String, Boolean> hasMoresForCates = new HashMap<String, Boolean>();
	
	static {
		for (String cate : MOVIE_CATES) {
			hasMoresForCates.put(cate, true);
		}
	}
	
	private YoutubeMovieSpider(){}
	
	public static int grabAllMovies(){
		//交替抓取每个分类		
		int count = 0;
		int page = 1;
		for (; page < MOVIE_MAX_PAGE; page++) {
			for (String cate : MOVIE_CATES) {
				if(hasMoresForCates.get(cate)){
					String url = StringUtils.format(MOVIE_LIST_URL_FORMAT, cate, page);
					int saved = parseAndSaveOnePageMovies(url, count);
					if(saved == 0){
						hasMoresForCates.put(cate, false);
					}else{
						count += saved;
					}
				}
				
			}
		}	
		
		return count;
	}

	private static int parseAndSaveOnePageMovies(String url, int index) {
		Document doc = getDocument(url);
		if(doc == null){
			return 0;
		}
		
		Elements lis = doc.select("#branded-page-body > div.channel-tab-content > div.tab-content-body > div.primary-pane > div.channel-browse > ul.channels-browse-content-grid > li.channels-content-item");
		if(lis.isEmpty()){
			return 0;
		}
		
		List<Movie> movies = new ArrayList<Movie>();
		for (Element ele : lis) {
			//image
			String img = ele.select("a.ux-thumb-wrap > span.video-thumb > span > span > img").attr("data-thumb");
			if(img.isEmpty()){
				continue;
			}			
			
			Elements a = ele.select("a.ux-thumb-wrap");			
			String href = a.attr("href");			
			String vid = URLUtils.parseQueryString(href).get("v");
			if(vid == null){
				//get from image: http://i4.ytimg.com/vi/sMHui1TGDxA/movieposter.jpg
				int ind = img.indexOf("/vi/");
				String s = img.substring(ind + 4);
				vid = s.substring(0, s.indexOf("/"));
			}
			
			String detailUrl = StringUtils.format(MOVIE_DETAIL_URL_FORMAT, vid);
			
			//genre, duration, director, actors, publish_time, desc
			Movie movie = grabMovieDetail(detailUrl);						
			
			if(movie == null){
				continue;
			}
			
			movie.setVideoId(vid);		
			
			String title = ele.select("a.content-item-title").text().trim();			
			movie.setTitle(title);
			movie.setTitleInitial(StringUtils.getInitial(title));				
					
			movie.setThumbnail(img);
			
			movie.setId(SiteConstants.ID_PREFIX_YOUTUBE + vid);
			movie.setWebsite(SiteConstants.WEBSITE_YOUTUBE);
			movie.setMovieType(Movie.TYPE_MOVIE);
			movie.setItemUrl(StringUtils.format(MOVIE_WATCH_URL_FORMAT, vid));
			
			
			movies.add(movie);			
		}		
		
		//save to db
		Date version = DateUtils.getNow();
		SqlSession session = DefaultDBHelper.getBatchSqlSession();
		try {
			MovieMapper mapper = session.getMapper(MovieMapper.class);
			MovieGenreMapper genreMapper = session.getMapper(MovieGenreMapper.class);
			for (Movie movie : movies) {
				index++;
				movie.setVersion(version);							
				String movieId = movie.getId();
				Movie temp = mapper.selectByPrimaryKey(movieId);
				if(temp != null){				
					Integer sortIndex = temp.getSortIndex();					
					if(sortIndex == null || sortIndex.intValue() > index){
						movie.setSortIndex(index);
					}else{
						movie.setSortIndex(sortIndex);
					}
					mapper.updateByPrimaryKey(movie);
				}else{
					//insert					
					movie.setSortIndex(index);
					mapper.insert(movie);
					
					//genre
					String genreNames = movie.getGenre();
					if( ! StringUtils.isNullOrEmpty(genreNames)){
						String[] genres = genreNames.split(",");
						for (String genre : genres) {
							QueryExample ex = new QueryExample();
							ex.or().addCriterion("movie_genre.movie_id=", movieId)
									.addCriterion("movie_genre.genre_name=", genre);
							
							if(genreMapper.countByExample(ex) == 0){
								//增加关联
								MovieGenre movieGenre = new MovieGenre();
								movieGenre.setId(UUID.randomUUID().toString());
								movieGenre.setMovieId(movieId);
								movieGenre.setGenreName(genre);
								genreMapper.insert(movieGenre);
							}
						}
					}
				}
			}
			
			session.commit();
		} finally {
			session.close();			
		}
		
		return movies.size();
	}

	private static Movie grabMovieDetail(String url) {		
		//image, desc, genre, duration, publish_time, director, actors		
		Document doc = getDocument(url);
		if(doc == null){
			return null;
		}		
		
		if(doc.select("#poster-container").isEmpty()){
			return null;
		}	
		
		Movie movie = new Movie();
		//genre, duration, publish_time
		Elements dts = doc.select("#browse-entity-side-column > div.browse-entity-section > dl > dt");
		for (Element dt : dts) {
			String type = dt.attr("itemprop");
			String value = dt.text().trim();
			if("genre".equals(type)){				
				//修正genre				
				movie.setGenre(YoutubeMovieGenres.remendAllGenres(value));				
			}else if("duration".equals(type)){
				movie.setDuration(StringUtils.toSeconds(value));
			}else if("datePublished".equals(type)){
				movie.setPublishTime(DateUtils.toDate(value));
			}
		}
		
		if(movie.getDuration() == null || movie.getDuration() == 0){
			return null;
		}
			
		String desc = doc.select("#browse-entity-main-column > div.browse-entity-section > div.browse-entity-section > p").text().trim();
		movie.setDescription(desc);		
		
		//director, actors
		Elements studio = doc.select("#browse-entity-side-column > div.studio-details");
		if( ! studio.isEmpty()){
			Elements slis = studio.select("ol.character-list > li");
			Iterator<Element> iters = slis.iterator();
			if(slis.size() > 4){
				iters = slis.subList(0, 4).iterator();				
			}
			StringBuilder sb = new StringBuilder();
			while (iters.hasNext()) {
				Element el = iters.next();
				String actor = el.select("span.actor-name").text().trim();
				if(actor.isEmpty()){
					actor = el.select("a.actor-name").text().trim();
				}
				sb.append(actor).append(",");
			}
			if(sb.length() > 1){
				movie.setActors(sb.substring(0, sb.length() - 1));
			}			
			
			Elements dlis = studio.select("ol > li");
			for (Element ele : dlis) {
				if("director".equals(ele.attr("itemprop"))){
					movie.setDirector(ele.text().trim());
					break;
				}
			}
		}
		
		//score
		String score = doc.select("#browse-entity-main-column > div.browse-entity-section > div.rottentomato-score-container > p.tomatometer-container").text().trim();
		if( ! score.isEmpty()){
			score = score.replace("%", "");
			movie.setScore(BigDecimal.valueOf((NumberUtils.parseInt(score))/10D));
		}
		
		return movie;
	}

	private static Document getDocument(String url) {
		try {
			return Jsoup.connect(url).timeout(CONNECT_READ_TIMEOUT).userAgent(USER_AGENT).referrer(REFERRER).get();
		} catch (IOException e) {
			logger.error("error happened when fetch YouTube webpage: " + e);
		}
		
		return null;
	}
	
	public static void main(String[] args) {
		grabAllMovies();
	}	
}
